{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import sed_eval"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>filename</th>\n",
       "      <th>onset</th>\n",
       "      <th>offset</th>\n",
       "      <th>event_label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>YSb0169-lqLs_0.000_9.000.wav</td>\n",
       "      <td>0.778</td>\n",
       "      <td>7.187</td>\n",
       "      <td>Dog</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Y9OqtuFGCCR8_30.000_40.000.wav</td>\n",
       "      <td>0.000</td>\n",
       "      <td>0.302</td>\n",
       "      <td>Alarm_bell_ringing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Y9OqtuFGCCR8_30.000_40.000.wav</td>\n",
       "      <td>0.789</td>\n",
       "      <td>1.126</td>\n",
       "      <td>Alarm_bell_ringing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Y9OqtuFGCCR8_30.000_40.000.wav</td>\n",
       "      <td>1.567</td>\n",
       "      <td>1.939</td>\n",
       "      <td>Alarm_bell_ringing</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Y9OqtuFGCCR8_30.000_40.000.wav</td>\n",
       "      <td>2.368</td>\n",
       "      <td>2.728</td>\n",
       "      <td>Alarm_bell_ringing</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         filename  onset  offset         event_label\n",
       "0    YSb0169-lqLs_0.000_9.000.wav  0.778   7.187                 Dog\n",
       "1  Y9OqtuFGCCR8_30.000_40.000.wav  0.000   0.302  Alarm_bell_ringing\n",
       "2  Y9OqtuFGCCR8_30.000_40.000.wav  0.789   1.126  Alarm_bell_ringing\n",
       "3  Y9OqtuFGCCR8_30.000_40.000.wav  1.567   1.939  Alarm_bell_ringing\n",
       "4  Y9OqtuFGCCR8_30.000_40.000.wav  2.368   2.728  Alarm_bell_ringing"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_path = \"../dataset/metadata/eval/eval.csv\"\n",
    "df_eval = pd.read_csv(eval_path, sep=\"\\t\", header=0)\n",
    "df_eval.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Following cell is taken from evaluation_measures.py, but added here to be explicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_event_list_current_file(df, fname):\n",
    "    \"\"\"\n",
    "    Get list of events for a given filename\n",
    "    :param df: pd.DataFrame, the dataframe to search on\n",
    "    :param fname: the filename to extract the value from the dataframe\n",
    "    :return: list of events (dictionaries) for the given filename\n",
    "    \"\"\"\n",
    "    event_file = df[df[\"filename\"].str.contains(fname)]\n",
    "    if len(event_file) == 1:\n",
    "        if pd.isna(event_file[\"event_label\"].iloc[0]):\n",
    "            event_list_for_current_file = [{\"filename\": fname}]\n",
    "        else:\n",
    "            event_list_for_current_file = event_file.to_dict('records')\n",
    "    else:\n",
    "        event_list_for_current_file = event_file.to_dict('records')\n",
    "\n",
    "    return event_list_for_current_file\n",
    "\n",
    "def event_based_evaluation_df(reference, estimated):\n",
    "    \"\"\"\n",
    "    Calculate EventBasedMetric given a reference and estimated dataframe\n",
    "    :param reference: pd.DataFrame containing \"filename\" \"onset\" \"offset\" and \"event_label\" columns which describe the\n",
    "    reference events\n",
    "    :param estimated: pd.DataFrame containing \"filename\" \"onset\" \"offset\" and \"event_label\" columns which describe the\n",
    "    estimated events to be compared with reference\n",
    "    :return: sed_eval.sound_event.EventBasedMetrics with the scores\n",
    "    \"\"\"\n",
    "    \n",
    "    evaluated_files = reference[\"filename\"].unique()\n",
    "    classes = []\n",
    "    classes.extend(reference.event_label.dropna().unique())\n",
    "    classes.extend(estimated.event_label.dropna().unique())\n",
    "    event_based_metric = sed_eval.sound_event.EventBasedMetrics(\n",
    "        event_label_list=classes,\n",
    "        t_collar=0.200,\n",
    "        percentage_of_length=0.2,\n",
    "        empty_system_output_handling='zero_score'\n",
    "    )\n",
    "\n",
    "    for fname in evaluated_files:\n",
    "        reference_event_list_for_current_file = get_event_list_current_file(reference, fname)\n",
    "        estimated_event_list_for_current_file = get_event_list_current_file(estimated, fname)\n",
    "\n",
    "        event_based_metric.evaluate(\n",
    "            reference_event_list=reference_event_list_for_current_file,\n",
    "            estimated_event_list=estimated_event_list_for_current_file,\n",
    "        )\n",
    "    return event_based_metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__init__() got an unexpected keyword argument 'empty_system_output_handling'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m--------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-77d2cf147b0c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mdf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"eval_results_baseline.csv\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msep\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"\\t\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mheader\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevent_based_evaluation_df\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdf_eval\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-7e823f53b02e>\u001b[0m in \u001b[0;36mevent_based_evaluation_df\u001b[0;34m(reference, estimated)\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mt_collar\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.200\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     36\u001b[0m         \u001b[0mpercentage_of_length\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0.2\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 37\u001b[0;31m         \u001b[0mempty_system_output_handling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'zero_score'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     38\u001b[0m     )\n\u001b[1;32m     39\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: __init__() got an unexpected keyword argument 'empty_system_output_handling'"
     ]
    }
   ],
   "source": [
    "df = pd.read_csv(\"eval_results_baseline.csv\", sep=\"\\t\", header=0)\n",
    "results = event_based_evaluation_df(df_eval, df)\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:dcase2018]",
   "language": "python",
   "name": "conda-env-dcase2018-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
